def test_sorting_year(self):
data = get_data(self.url)
self.driver.get(self.url)
# get dropdown for selecting filtering
select = Select(self.driver.find_element_by_id('lister-sort-by-options'))
select.select_by_value('us:descending') # Select Release Date filter
current_url = self.driver.current_url
# Getting years list of current url
current_url_years = get_year(current_url)
data = get_year(self.url)
data.sort(key = lambda x: -int(x.replace('(', '').replace(')', '')))
self.assertEqual(current_url_years, data,
'Sorting by release date does not work')
def credential_counter(data):
type = "credential.count"
date = utils.get_year(data['timestamp'])
identifier = data['identifier']
dt = coll_cred.find({
"type": type,
"date": date,
"identifier": identifier
}).count()
if dt == 0:
newdata = CredentialCount(None, data)
coll_cred.insert(newdata.to_mongo())
else:
mongodata = coll_metrics.find_one({
"type": type,
"date": date,
"identifier": identifier
})
existdata = CredentialCount(mongodata, data)
if existdata.checkNone():
pass
else:
coll_cred.update(
{
"type": type,
"date": date,
"identifier": identifier
}, existdata.to_update(), True, False)
def test_sorting_button_click(self):
self.driver.get(self.url)
select = Select(
self.driver.find_element_by_id('lister-sort-by-options')
) # get dropdown for selecting filtering
select.select_by_value('us:descending') # Select Release Date filter
sort_button = self.driver.find_element_by_xpath(
'//*[@id="main"]/div/span/div/div/div[3]/div/div/div[1]/span')
sort_button.click() # click sorting button
current_url = self.driver.current_url
# Getting years list of current url
current_url_years = get_year(current_url)
data = get_year(self.url)
data.sort(key=lambda x: int(x.replace('(', '').replace(')', '')))
self.assertEqual(current_url_years, data,
'Sorting button does not work properly')
def publication_year(self):
try:
pub_data = self.results[0].select('.publisher')[0].strings
except IndexError:
return None
for s in pub_data:
y = get_year(s)
if y and y >= self.min_year:
return y
return None
def main():
tm = TrelloManager()
sm = GoogleSheetManager()
rm = ReleaseManager(trello_manager=tm, sheet_manager=sm)
today = date.today()
rm.release(week=get_week(today),
quarter=get_quarter(today),
year=get_year(today),
dry_run=False,
)
def __init__(self,mongodata,data):
self._type = "credential.count"
self.mongodata = mongodata
self.data = data
self._identifier = data["identifier"]
self._date = utils.get_year(data['timestamp']) if mongodata is None else mongodata['date']
self._username_list = {} if mongodata is None or "username_list" not in mongodata else mongodata["username_list"]
self._password_list = {} if mongodata is None or "password_list" not in mongodata else mongodata["password_list"]
self._newdata = {}
self._incdata = {}
self._setdata = {}
if mongodata is None:
self._newdata = self._init_data()
else:
self._compute()
def release(self, week=None, quarter=None, year=None, dry_run=False):
new_archive = self._create_lists(week, quarter, year, dry_run)
completed = self._get_list_by_name("Completed")
self._move_all_cards(completed, new_archive, dry_run)
self._position_list(new_archive, completed, dry_run)
on_deck = self._get_list_by_name("On Deck")
self._create_weekly_cards(on_deck, dry_run)
icebox = self._get_list_by_name("Icebox")
date_next_week = date.today() + datetime.timedelta(weeks=2)
self._create_card_in_list(icebox,
"{}Q{} W{} Goal Ideas".format(
get_year(date_next_week),
get_quarter(date_next_week),
get_week(date_next_week)),
checklist=True,
dry_run=dry_run,
checklist_name="Goal Ideas")
def bugs_fixed_in_year(person, **kw):
kw = kw.copy()
timeframe = get_year(YEAR)
kw.update({
'emailtype1': 'substring',
'emailassigned_to1': 1,
'email1': person['bugzilla_email'],
'f1': 'resolution',
'o1': 'changedto',
'v1': 'FIXED',
'f2': 'resolution',
'o2': 'changedafter',
'v2': timeframe['start'],
'f3': 'resolution',
'o3': 'changedbefore',
'v3': timeframe['end'],
})
return kw
def create_listitem(episode):
"""Create a xbmcgui.ListItem from provided episode details"""
kwargs = {
'label': episode.get('title', ''),
'label2': '',
'path': episode.get('file', '')
}
if utils.supports_python_api(18):
kwargs['offscreen'] = True
listitem = xbmcgui.ListItem(**kwargs)
listitem.setInfo(type='Video',
infoLabels={
'dbid': episode.get('episodeid',
constants.UNKNOWN_DATA),
'path': episode.get('file', ''),
'title': episode.get('title', ''),
'plot': episode.get('plot', ''),
'tvshowtitle': episode.get('showtitle', ''),
'season': episode.get('season',
constants.UNKNOWN_DATA),
'episode': episode.get('episode',
constants.UNKNOWN_DATA),
'rating': str(float(episode.get('rating', 0.0))),
'aired': episode.get('firstaired', ''),
'premiered': episode.get('firstaired', ''),
'year': utils.get_year(episode.get('firstaired', '')),
'dateadded': episode.get('dateadded', ''),
'lastplayed': episode.get('lastplayed', ''),
'playcount': episode.get('playcount', 0),
'mediatype': 'episode'
})
listitem.setProperty('tvshowid',
str(episode.get('tvshowid', constants.UNKNOWN_DATA)))
listitem.setArt(episode.get('art', {}))
listitem.setProperty('isPlayable', 'true')
listitem.setPath(episode.get('file', ''))
if utils.supports_python_api(18):
listitem.setIsFolder(False)
return listitem
# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení GeoPython pro začátečníky'
copyright = u'2014-%d GISMentors' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.5beta'
# The full version, including alpha/beta/rc tags.
release = '%s' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'Skoleni-GeoPython'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení GRASS GIS pro začátečníky'
copyright = u'2014-%d, Martin Landa a Jáchym Čepický (GISMentors)' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.1.0beta'
# The full version, including alpha/beta/rc tags.
release = '%s' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'skoleni-grass-gis-zacatecnik'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join("..", "sphinx-template"))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u"Školení QGIS pro začátečníky"
copyright = u"2014-%d GISMentors" % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = "0.9"
# The full version, including alpha/beta/rc tags.
release = "%sbeta" % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = "skoleni-qgis-zacatecnik"
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'GISMentors: GRASS GIS Workshop in Jena'
copyright = u'%d GISMentors.eu' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.0'
# The full version, including alpha/beta/rc tags.
release = '%s' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'grass-gis-workshop-jena-2018-gismentors'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení GeoServer pro začátečníky'
copyright = u'2016-%d GISMentors' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '%salfa' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'skoleni-geoserver-zacatecnik'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení QGIS pro začátečníky'
copyright = u'2014-%d, Alžbeta Gardoňová, Ľudmila Furtkevičová, Oto Kaláb, Vojtěch Dubrovský (GISMentors.eu)' % get_year(
)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.9'
# The full version, including alpha/beta/rc tags.
release = '%sbeta' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'skoleni-qgis-zacatecnik'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení GeoServer pro začátečníky'
copyright = u'2016-%d, Jan Růžička (GISMentors.eu)' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.9'
# The full version, including alpha/beta/rc tags.
release = '%sbeta' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'skoleni-geoserver-zacatecnik'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
# -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Jena GRASS GIS Workshop'
copyright = u'2018-%d Martin Landa and GISMentors.eu' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '2.0'
# The full version, including alpha/beta/rc tags.
release = '%s' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'grass-gis-workshop-jena'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
idx = sys.argv[3]
sd = datetime.datetime.strptime(sd, '%Y%m%d')
ed = datetime.datetime.strptime(ed, '%Y%m%d')
rics = pd.read_csv(root_path + '/data/compo/crypto/{}.txt'.format(idx))
rics = list(rics.ric.unique())
#rics = ['BTC']
funding_path = root_path + 'data/funding/{}/'.format(idx)
os.makedirs(funding_path, 0o777, exist_ok=True)
s = Session()
ts = int(time.time() * 1000)
for d in [sd + datetime.timedelta(x) for x in range((ed - sd).days + 1)]:
d = d.strftime('%Y%m%d')
df = pd.DataFrame()
st = datetime.datetime(int(utils.get_year(d)), int(utils.get_month(d)),
int(utils.get_day(d)), 0, 0)
et = datetime.datetime(int(utils.get_year(d)), int(utils.get_month(d)),
int(utils.get_day(d)), 23, 59)
print('{} {}'.format(st, et))
st = calendar.timegm(st.timetuple())
et = calendar.timegm(et.timetuple())
for ric in rics:
print(ric)
request = Request(
'GET',
'https://ftx.com/api/funding_rates?start_time={}&end_time={}&future={}-PERP'
.format(st, et, ric))
prepared = request.prepare()
signature_payload = f'{ts}{prepared.method}{prepared.path_url}'.encode(
)
def publication_year(self):
y = get_year(self.query)
if y and y >= self.min_year:
return y
def publication_year(self):
y = self._get_volume_info('publishedDate')
return get_year(y) if isinstance(y, str) else y
import json
import concurrent.futures
from utils import get_year, get_month
from utils import get_submitter_info, get_country_from_domain
info = []
year_low = 2020
month_low = 8
print("Reading data...")
with open("data/arxiv.json") as f:
for line in f:
data = json.loads(line)
year = get_year(data["versions"])
month = get_month(data["versions"])
if year >= year_low and month >= month_low:
info.append((data["submitter"], data["title"]))
print("Done. Number of articles:", len(info))
def task(id, info_slice):
print(f"Worker {id} started")
already_seen_domains = {}
for submitter, title in info_slice:
country = None
try:
submitter_info = get_submitter_info(submitter, title)
import datetime, time
import sys
import os
import pandas as pd
import calendar
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
if len(sys.argv) < 4:
print("usage: ./dump_glassnode.py date endpoint metric")
quit()
dt = sys.argv[1]
endpoint = sys.argv[2]
metric = sys.argv[3]
yy = int(utils.get_year(dt))
mm = int(utils.get_month(dt))
dd = int(utils.get_day(dt))
sd = datetime.datetime(yy, mm, dd, 0, 0)
ed = sd + datetime.timedelta(1)
print(dt)
sd = calendar.timegm(sd.timetuple())
ed = calendar.timegm(ed.timetuple())
rics = pd.read_csv(root_path + '/data/compo/crypto/ftx.txt')
rics = list(rics.ric.unique())
#rics = ['btc', 'eth']
data_path = root_path + 'data/glassnode/{}/{}/'.format(endpoint, metric)
os.makedirs(data_path, 0o777, exist_ok=True)
df = pd.DataFrame()
def publication_year(self):
if self.response:
y = get_year(self.response['bib']['pub_year'])
return y if y >= self.min_year else None
# -*- coding: utf-8 -*-
import sys
import os
sys.path.append(os.path.join('..', 'sphinx-template'))
from utils import get_month_year, get_year
# -- General configuration ------------------------------------------------
# General information about the project.
project = u'Školení GeoPython'
copyright = u'2014-%d, Jáchym Čepický a Martin Landa (GISMentors.eu)' % get_year()
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.2beta'
# The full version, including alpha/beta/rc tags.
release = '%s' % version
# -- Options for HTML output ----------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'Skoleni-GeoPython'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = project
def retrieve_data(workbook, pattern, verbose=False):
wb = load_workbook(workbook, read_only=True)
sheets = [n for n in wb.sheetnames if re.search(pattern, n, flags=re.I)]
out_headers = []
out_data = OrderedDict()
for sheet_i, sheet_name in enumerate(sheets):
if verbose:
print(' Sheet "{0}"...'.format(sheet_name))
sheet = wb[sheet_name]
assert isinstance(sheet, ReadOnlyWorksheet), 'Found type {0}'.format(
type(sheet))
year_row, _, header_rows = utils.get_header_rows(sheet)
years, headers, columns = utils.get_input_headers(sheet)
default_cvt = convert.get_default_converter_by_sheet(sheet)
# headers
curr_headers = convert.get_headers(sheet)
if sheet_i == 0:
out_headers = curr_headers
else:
if curr_headers != out_headers:
warnings.warn('Differences:\n {0}'.format(
'\n '.join(set(out_headers) - set(curr_headers))))
def get_col(h, yr):
d = columns[h]
if isinstance(yr, list):
if yr:
yr = yr[0]
else:
yr = 0
else:
yr = int(yr)
return d[yr] if yr in d else next(v for v in d.values())
char_col = get_col(headers[0], years)
check_col = get_col(headers[1], years)
cvt = default_cvt
year = 0
characteristic = ''
sub_char = ''
if year_row:
gen_years = years
else:
gen_years = [None]
for y in gen_years:
for row in itertools.islice(sheet.rows, max(header_rows), None):
if not year_row:
year = utils.get_year(sheet, row, default=year)
y = year
check_val = utils.prepare_str(row[check_col].value)
char_val = utils.prepare_str(row[char_col].value)
if char_val:
sub_char = char_val
if utils.is_number(check_val) or check_val in {
'*', '†', 'n.a.'
}:
key = '__'.join([
str(y), characteristic,
utils.prepare_str(row[char_col].value)
])
if key not in out_data:
out_data[key] = {'Sub-Characteristic': sub_char}
if characteristic and 'Characteristic' in out_headers:
out_data[key]['Characteristic'] = characteristic
if y:
out_data[key]['Year'] = str(y)
# add data to row
out_data[key].update({
new_h: c(row[get_col(h, y)].value)
for h, new_h, c in cvt.iter_headers(headers[1:])
})
else:
characteristic = char_val
if not characteristic and check_val:
cvt = convert.DataTypeConverter(check_val,
default=default_cvt)
return out_headers, out_data
import sys
import os
import pandas as pd
import calendar
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../../'
sys.path.append(root_path + 'scripts/')
import utils
if len(sys.argv) < 3:
print("usage: ./dump_erd_crypto.py date idx")
quit()
sd = sys.argv[1]
ed = sys.argv[2]
idx = sys.argv[3]
sd = datetime.datetime(int(utils.get_year(sd)), int(utils.get_month(sd)),
int(utils.get_day(sd)), 0, 0)
ed = datetime.datetime(int(utils.get_year(ed)), int(utils.get_month(ed)),
int(utils.get_day(ed)), 0, 0)
print('{} {}'.format(sd, ed))
sd = calendar.timegm(sd.timetuple())
ed = calendar.timegm(ed.timetuple())
#sd = int(time.mktime(sd.timetuple()))
#ed = int(time.mktime(ed.timetuple()))
rics = pd.read_csv(root_path + '/data/compo/crypto/{}.txt'.format(idx))
rics = list(rics.ric.unique())
erd_path = root_path + 'data/erd/{}/'.format(idx)
s = Session()
ts = int(time.time() * 1000)
df = pd.DataFrame()